Classifying Book Genres Based On Review Text Using Machine Learning¶

## IMPORT DATA

# Imports
import pandas as pd

# Load a sample of reviews to handle memory efficiently
df_reviews = pd.read_json(
    '/fs/ess/PAS2038/PHYSICS_5680_OSU/project_data/goodreads/goodreads/complete/goodreads_reviews_dedup.json.gz', 
    nrows=100000, 
    lines=True
)

# Extract book_ids from the reviews
book_ids = set(df_reviews['book_id'].unique())

# Initialize a list to store filtered genre data chunks
filtered_genre_chunks = []

# Process genres data in chunks
for chunk in pd.read_json(
    '/fs/ess/PAS2038/PHYSICS_5680_OSU/project_data/goodreads/goodreads/complete/goodreads_book_genres_initial.json.gz', 
    lines=True, 
    chunksize=10000
):
    # Filter for rows with matching book_ids and non-empty genres
    filtered_chunk = chunk[chunk['book_id'].isin(book_ids) & chunk['genres'].notna() & (chunk['genres'].str.len() > 0)]
    
    # Filter to make sure that the first genre is not 'fiction' unless a second genre exists
    filtered_chunk = filtered_chunk[filtered_chunk['genres'].apply(lambda x: len(x) > 1 if 'fiction' in x else True)]
    
    filtered_genre_chunks.append(filtered_chunk)

# Combine all filtered chunks into one dataframe
df_genres = pd.concat(filtered_genre_chunks, ignore_index=True)

# Merge reviews and genres on book_id
df = pd.merge(df_reviews, df_genres, on='book_id', how='inner')

# Check the shape and missing data of the final merged dataframe
print('Shape of merged dataframe:', df.shape)
print()
print('Missing data summary:\n', df.isnull().sum())
print()
print('Number of rows with null genres:', df_genres['genres'].isnull().sum())

df.columns

df.head()

## Distribution of genres

# Imports
import plotly.express as px

# Define a function to get the first genre
def get_first_genre(genre_dict):
    if isinstance(genre_dict, dict) and genre_dict:  # Check if it's a non-empty dictionary
        return next(iter(genre_dict.keys()))  # Get the first key (genre)
    return None  # Return None if it's not a dictionary or is empty

# Apply the function to create the primary genre column
df['primary_genre'] = df['genres'].apply(get_first_genre)

# Count of primary genres
genre_counts = df['primary_genre'].value_counts().reset_index()
genre_counts.columns = ['primary_genre', 'count']

# Plot the top genres
fig = px.bar(
    genre_counts.head(10), 
    x='primary_genre', 
    y='count', 
    title='Top 10 Genres in Dataset',
    labels={'primary_genre': 'Genre', 'count': 'Count'},
    color='count',
    color_continuous_scale='Viridis'
)

# Update layout
fig.update_layout(
    xaxis_title='Genre', 
    yaxis_title='Count', 
    xaxis_tickangle=-45,
    width=900,  # Increase figure width
    height=500, # Increase figure height
    margin=dict(l=60, r=60, t=60, b=100)  # Adjust margins for more spacing
)

fig.show()

## Distribution of secondary genres

# Define a function to get the second genre if it exists
def get_second_genre(genre_dict):
    if isinstance(genre_dict, dict) and len(genre_dict) > 1:  # Check if it's a dictionary with at least two items
        return list(genre_dict.keys())[1]  # Get the second key (subgenre)
    return None  # Return None if it's not a dictionary or doesn't have a second genre

# Apply the function to create the subgenre column
df['secondary_genre'] = df['genres'].apply(get_second_genre)

# Count of subgenres
secondary_genre_counts = df['secondary_genre'].value_counts().reset_index()
secondary_genre_counts.columns = ['secondary_genre', 'count']

# Plot the top subgenres
fig = px.bar(
    secondary_genre_counts.head(10), 
    x='secondary_genre', 
    y='count', 
    title='Top 10 Subgenres in Dataset',
    labels={'secondary_genre': 'Secondary Genre', 'count': 'Count'},
    color='count',
    color_continuous_scale='Viridis'
)

# Update layout
fig.update_layout(
    xaxis_title='Secondary Genre', 
    yaxis_title='Count', 
    xaxis_tickangle=-45,
    width=900,  
    height=500, 
    margin=dict(l=60, r=60, t=60, b=100) 
)

fig.show()

## Distribution of third genres, if they exist

# Function to get the third genre, if it exists
def get_third_genre(genre_dict):
    if isinstance(genre_dict, dict) and len(genre_dict) >= 3:  # Check if it's a dictionary with at least 3 genres
        return list(genre_dict.keys())[2]  # Get the third key (genre)
    return None  # Return None if there are fewer than 3 genres

# Apply the function to create the third genre column
df['third_genre'] = df['genres'].apply(get_third_genre)

# Count of third genres
third_genre_counts = df['third_genre'].value_counts().reset_index()
third_genre_counts.columns = ['third_genre', 'count']

# Plot the top third genres
fig = px.bar(third_genre_counts.head(10), 
             x='third_genre', 
             y='count', 
             title='Top 10 Third Genres in Dataset',
             labels={'third_genre': 'Third Genre', 'count': 'Count'},
             color='count',
             color_continuous_scale='Viridis')

# Update layout
fig.update_layout(
    xaxis_title='Third Genre', 
    yaxis_title='Count', 
    xaxis_tickangle=-45,
    width=900,  
    height=500,
    margin=dict(l=60, r=60, t=60, b=100)  
)

fig.show()

## Average rating for each genre

# Calculate average rating for each genre
avg_rating_by_genre = df.groupby('primary_genre')['rating'].mean().reset_index().sort_values(by='rating', ascending=False)

# Plot average rating by genre
fig = px.bar(avg_rating_by_genre.head(10), 
             x='primary_genre', 
             y='rating', 
             title='Top 10 Genres by Average Rating',
             labels={'primary_genre': 'Genre', 'rating': 'Average Rating'},
             color='rating',
             color_continuous_scale='purp')  

# Update layout
fig.update_layout(
    xaxis_title='Genre', 
    yaxis_title='Average Rating', 
    xaxis_tickangle=-45,
    width=800, 
    height=500,  
    margin=dict(l=40, r=40, t=60, b=100) 
)

fig.show()

## Distribution of ratings

# Plot distribution of ratings
fig = px.histogram(df, 
                   x='rating', 
                   nbins=10, 
                   title='Distribution of Ratings',
                   labels={'rating': 'Rating'},
                   color_discrete_sequence=['skyblue'])

# Update traces 
fig.update_traces(opacity=0.85)  

# Update layout
fig.update_layout(
    xaxis_title='Rating', 
    yaxis_title='Count',
    width=800,  
    height=500, 
    title_x=0.5, 
    xaxis=dict(gridcolor='lightgray'),  
    yaxis=dict(gridcolor='lightgray'),
    margin=dict(l=40, r=40, t=60, b=100)  
)

fig.show()

## Average length of review by genre

# Add a column for word count in reviews
df['review_word_count'] = df['review_text'].apply(lambda x: len(str(x).split()))

# Plot average review word count by genre
avg_word_count_by_genre = df.groupby('primary_genre')['review_word_count'].mean().reset_index().sort_values(by='review_word_count', ascending=False)
fig = px.bar(avg_word_count_by_genre.head(10), 
             x='primary_genre', 
             y='review_word_count', 
             title='Average Review Word Count by Genre',
             labels={'primary_genre': 'Genre', 'review_word_count': 'Average Word Count'},
             color='review_word_count',
             color_continuous_scale='Sunsetdark')

# Update layout 
fig.update_traces(opacity=0.85) 
fig.update_layout(
    xaxis_title='Genre', 
    yaxis_title='Average Word Count',
    title_x=0.5,  
    width=800,  
    height=500,
    xaxis=dict(tickangle=-45, title_standoff=10),  
    margin=dict(l=40, r=40, t=60, b=100)  
)

fig.show()

## REPLACE THE FIRST GENRE 'FICTION' WITH ITS SECONDARY GENRE

# Imports
from collections import Counter

# Count the number of books where the primary genre is 'fiction' and the secondary genre is None (or empty)
num_fiction_with_no_secondary_genre = df[(df['primary_genre'] == 'fiction') & 
                                          (df['secondary_genre'].isna() | (df['secondary_genre'] == ''))].shape[0]
print(f"Number of 'fiction' books with no secondary genre: {num_fiction_with_no_secondary_genre}")
print()

# Replace the primary genre with the secondary genre where the primary genre is 'fiction'
df.loc[df['primary_genre'] == 'fiction', 'primary_genre'] = df['secondary_genre']

# Check new genres
genre_counts = Counter(df['primary_genre'])
for genre, count in genre_counts.items():
    print(f"{genre}: {count}")
print()

# Check for None or NaN values in the primary_genre column
nan_rows = df[df['primary_genre'].isnull()]
print(f"Number of rows with missing primary genre: {nan_rows.shape[0]}")

## CLEAN REVIEW DATA

# Imports
import nltk
import re
import pandas as pd
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from nltk.tokenize import word_tokenize
nltk.download('stopwords')
nltk.download('wordnet')
nltk.download('omw-1.4')
nltk.download('punkt')

# Define the stop words set
stop_words = set(stopwords.words('english'))

# Initialize the lemmatizer
lemmatizer = WordNetLemmatizer()

# Preprocessing function
def preprocess_review(review_text):
    
    # Lowercase and tokenize
    tokens = word_tokenize(review_text.lower())
    
    # Remove stopwords
    tokens = [word for word in tokens if word not in stop_words]
    
    # Lemmatize tokens
    tokens = [lemmatizer.lemmatize(word) for word in tokens]
    
    # Remove non-alphanumeric characters
    tokens = [re.sub(r'\W+', '', token) for token in tokens if re.sub(r'\W+', '', token) != '']
    
    # Remove numbers
    tokens = [token for token in tokens if not token.isdigit()]
    
    # Remove short words (length <= 2)
    tokens = [token for token in tokens if len(token) > 2]
    
    return tokens

# Apply the preprocessing function to the reviews
df['final_reviews'] = df['review_text'].apply(preprocess_review)

# Drop duplicates
df = df.drop_duplicates(subset=['final_reviews'])

# View a result
print(df.final_reviews[0], '\n')

## TF-IDF VECTORIZATION

# Imports
from sklearn.feature_extraction.text import TfidfVectorizer

# Initialize the vectorizer
vectorizer = TfidfVectorizer(max_features=5000)  # Set a limit on the number of features

# Apply TF-IDF vectorization to final_reviews
df['review_text_str'] = df['final_reviews'].apply(lambda x: ' '.join(x))

# Fit and transform the reviews
tfidf_matrix = vectorizer.fit_transform(df['review_text_str'])

# Convert the TF-IDF matrix to a df for inspection
tfidf_df = pd.DataFrame(tfidf_matrix.toarray(), columns=vectorizer.get_feature_names_out())
print('Shape of the TF-IDF matrix:', tfidf_matrix.shape)
print()
print(f'TF-IDF DataFrame head:\n{tfidf_df.head()}')

## SENTIMENT ANALYSIS

# Imports
import numpy as np
from nltk.sentiment.vader import SentimentIntensityAnalyzer
nltk.download('vader_lexicon')

# Initialize sentiment analyzer
sia = SentimentIntensityAnalyzer()

# Get sentiment score
def get_sentiment_score(tokens):
    # Check if tokens is empty or None
    if not tokens or tokens is None:
        return np.nan  # Return NaN if no valid tokens
    # Join tokens back into a string for sentiment analysis    
    review_text = ' '.join(tokens)
    sentiment_score = sia.polarity_scores(review_text)
    return sentiment_score['compound']

# Apply sentiment analysis to the 'final_reviews' column
df['sentiment_score'] = df['final_reviews'].apply(get_sentiment_score)

# Verify the results
print(df[['final_reviews', 'sentiment_score']].head())

## Distribution of sentiment scores

# Plot histogram of sentiment scores
fig = px.histogram(df, x='sentiment_score', nbins=30, title='Distribution of Sentiment Scores', 
                   labels={'sentiment_score': 'Sentiment Score'})
fig.update_layout(xaxis_title='Sentiment Score', yaxis_title='Frequency')
fig.show()

## Average sentiment score per primary genre

# Calculate average sentiment score for each genre
avg_sentiment_by_genre = df.groupby('primary_genre')['sentiment_score'].mean().reset_index()

# Bar chart for average sentiment by genre
fig = px.bar(avg_sentiment_by_genre, x='primary_genre', y='sentiment_score', 
             title='Average Sentiment Score by Genre',
             labels={'primary_genre': 'Primary Genre', 'sentiment_score': 'Average Sentiment Score'}, 
             color='sentiment_score', 
             color_continuous_scale='tealgrn')

# Update layout
fig.update_traces(opacity=0.85) 
fig.update_layout(
    xaxis_title='Primary Genre', 
    yaxis_title='Average Sentiment Score',
    title_x=0.5,
    width=800,  
    height=500,  
    xaxis=dict(tickangle=-45, title_standoff=10),  
    margin=dict(l=40, r=40, t=60, b=100)  
)

fig.show()

## Sentiment vs review length

# Calculate review length
df['review_length'] = df['final_reviews'].apply(len)

# Scatter plot of sentiment vs. review length
fig = px.scatter(df, x='review_length', y='sentiment_score', 
                 title='Sentiment Score vs. Review Length',
                 labels={'review_length': 'Review Length', 'sentiment_score': 'Sentiment Score'},
                 opacity=0.6,
                 color='sentiment_score',
                 color_continuous_scale='spectral')

# Update layout for better readability
fig.update_traces(opacity=0.85)  
fig.update_layout(
    xaxis_title='Review Length', 
    yaxis_title='Sentiment Score',
    title_x=0.5,  
    width=800,  
    height=500,  
    xaxis=dict(tickangle=-45, title_standoff=10),  
    margin=dict(l=40, r=40, t=60, b=100)  
)

fig.show()

## CREATE AND TRAIN MODEL

# Imports
from sklearn.ensemble import RandomForestClassifier

# Initialize the model
model = RandomForestClassifier(
    n_estimators=500,       # Number of trees
    max_depth=None,          # Allow trees to grow as deep as they want
    min_samples_split=2,     # Minimum samples to split
    min_samples_leaf=4,      # Minimum samples at leaf
    max_features='log2',     # Number of features to consider at each split
    bootstrap=True,          # Use bootstrap sampling
    criterion='gini',       # Quality of split
    random_state=42          # For reproducibility
)

# Train the model
model.fit(X_train, y_train)

## CLASSIFICATION REPORT AND CONFUSION MATRIX

# Imports
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
from sklearn.metrics import classification_report, accuracy_score
import matplotlib.pyplot as plt

# Make predictions on the test set
y_pred = model.predict(X_test)

# Evaluate the model performance
print('Accuracy:', accuracy_score(y_test, y_pred))
print()
print('Classification Report:\n', classification_report(y_test, y_pred, target_names=le.classes_))

# Generate the confusion matrix
cm = confusion_matrix(y_test, y_pred)

# Create a confusion matrix display
disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=le.classes_)

# Customize the plot
fig, ax = plt.subplots(figsize=(8,8))
disp.plot(ax=ax, xticks_rotation='vertical')

# Show the plot
plt.title("Confusion Matrix")
plt.tight_layout()
plt.show()

## ROC Curve

# Imports
from sklearn.metrics import roc_curve, auc

# Compute ROC curve
fpr, tpr, thresholds = roc_curve(y_test, model.predict_proba(X_test)[:, 1], pos_label=1)

# Compute AUC
roc_auc = auc(fpr, tpr)

# Plot ROC curve
fig_roc = px.area(x=fpr, y=tpr, 
                  title=f'ROC Curve (AUC = {roc_auc:.2f})', 
                  labels={'x': 'False Positive Rate', 'y': 'True Positive Rate'},
                  height=400)
fig_roc.update_traces(fillcolor="rgba(157, 211, 144, 0.4)", 
                      line_color="rgba(157, 211, 144, 1)", 
                      opacity=0.4)


# Add random classifier
fig_roc.add_scatter(x=[0, 1], y=[0, 1], mode='lines', name='Random Classifier', line=dict(dash='dash', color='rgb(252, 115, 212)'))

fig_roc.show()

## Precision Recall Curve

# Imports
from sklearn.metrics import precision_recall_curve
from sklearn.preprocessing import label_binarize
import plotly.express as px

# Binarize the labels for multiclass precision-recall computation
y_test_bin = label_binarize(y_test, classes=range(len(le.classes_)))

# Initialize the plot
fig_pr = px.area(title='Precision-Recall Curve', 
                 labels={'x': 'Recall', 'y': 'Precision'}, 
                 height=600)

# Loop through each class to calculate precision, recall, and thresholds
for i in range(y_test_bin.shape[1]):  # Iterate over each class
    precision, recall, _ = precision_recall_curve(y_test_bin[:, i], model.predict_proba(X_test)[:, i])
    
    # Add each class curve to the plot
    fig_pr.add_scatter(x=recall, y=precision, mode='lines', name=f'{le.classes_[i]}')

# Update axis labels 
fig_pr.update_layout(
    xaxis_title='Recall',
    yaxis_title='Precision',
    height=600
)

fig_pr.show()

## Feature Importance

# Get the feature importances
importances = model.feature_importances_

# Get the feature names (these correspond to the TF-IDF features)
features = vectorizer.get_feature_names_out()

# Create a df for feature importance
feat_importance_df = pd.DataFrame({'Feature': features, 'Importance': importances})

# Sort the features by importance
feat_importance_df = feat_importance_df.sort_values(by='Importance', ascending=False)

# Plot feature importances
fig_feat_importance = px.bar(feat_importance_df.head(20), 
                             x='Feature', 
                             y='Importance', 
                             title='Top 20 Features', 
                             labels={'Feature': 'Feature', 'Importance': 'Importance'},
                             height=400,
                             color='Importance',
                             color_continuous_scale='tealgrn',  
                             range_color=[0, max(feat_importance_df['Importance'])]
                             )

# Update layourt
fig_feat_importance.update_layout(
    xaxis_tickangle=45,  
    xaxis={'tickfont': {'size': 12}},  
    yaxis_title='Importance', 
    yaxis={'tickfont': {'size': 14}}, 
    coloraxis_colorbar=dict(title='Importance'),  
    title={'font': {'size': 16}}, 
    showlegend=False  
)

fig_feat_importance.show()